rm(list = ls())

library(dplyr)
library(tidyr)
library(stringr)
library(varhandle)
library(nnet)
library(NeuralNetTools)
library(pROC)
library(gplots)
library(foreach)
library(parallel)
library(iterators)
library(doParallel)

Dataset: https://www.kaggle.com/datasnaek/chess

games = read.csv("games.csv") %>% distinct(id, .keep_all = TRUE)
ds.size.orig = dim(games)

# Eliminate irrelevant variables:
# game ID, player ID
# rated game
# opening ECO code
# start and end time (bad data, insufficient precision)
# moves (I feel just analyzing strings here would not go very far)
games = games %>% select(-c("id", "rated", "white_id", "black_id", "opening_eco", "created_at", "last_move_at", "moves", "victory_status"))

# increment_code is two different things (together defining the "speed" of the game):
# - initial assigned time for the game (minutes)
# - time increment after each move (seconds)
# so let's split that column
times_temp = as.data.frame(str_split_fixed(games$increment_code, "\\+", 2))
games = cbind(games, times_temp) %>% rename(initial_time = V1) %>% rename(time_increment = V2) %>% select(-c("increment_code"))
games$initial_time = unfactor(games$initial_time)
games$time_increment = unfactor(games$time_increment)

# eliminate rare openings
# we only consider openings that appear in a minimum number of games
op.count.min = 100
ocount = as.vector(games %>% group_by(opening_name) %>% summarize(count = n()) %>% arrange(desc(count)))
ops.orig = dim(ocount)
ocount = ocount %>% filter(count >= op.count.min)
ops.kept = dim(ocount)
write.csv(ocount, file = "ocount.csv", row.names = F)

games = games %>% filter(opening_name %in% unfactor(ocount$opening_name))
ds.size.pruned = dim(games)

print(paste("size of original dataset:", ds.size.orig[1], "games."), quote = F)
## [1] size of original dataset: 19113 games.
print(paste("We only consider games with openings that occur more than", op.count.min, "times in the dataset."), quote = F)
## [1] We only consider games with openings that occur more than 100 times in the dataset.
print(paste("Total number of openings in the dataset:", ops.orig[1]), quote = F)
## [1] Total number of openings in the dataset: 1477
print(paste("Number of openings kept:", ops.kept[1]), quote = F)
## [1] Number of openings kept: 39
print(paste("Size of dataset after pruning of openings:", ds.size.pruned[1]), quote = F)
## [1] Size of dataset after pruning of openings: 6506
print(paste("After pruning, the dataset is", round(100 * ds.size.pruned[1] / ds.size.orig[1]), "% the size of the original."), quote = F)
## [1] After pruning, the dataset is 34 % the size of the original.
# remove draws. Two reasons:
# - we're aiming to win
# - multinomial regression is extremely slow at this size, so we only want 2 levels for the outcome
games = games %>% filter(winner != "draw") %>% droplevels()

print(paste("Size of dataset after removing draws:", dim(games)[1]), quote = F)
## [1] Size of dataset after removing draws: 6220
games$win.bin = rep(0, times = dim(games)[1])
games$win.bin[which(games$winner == "white")] = 1
games$win.bin = as.factor(games$win.bin)
games = games %>% select(-c("winner"))

summary(games)
##      turns         white_rating   black_rating 
##  Min.   :  1.00   Min.   : 793   Min.   : 796  
##  1st Qu.: 36.00   1st Qu.:1353   1st Qu.:1353  
##  Median : 55.00   Median :1509   Median :1504  
##  Mean   : 59.03   Mean   :1532   Mean   :1528  
##  3rd Qu.: 77.00   3rd Qu.:1703   3rd Qu.:1701  
##  Max.   :222.00   Max.   :2621   Max.   :2516  
##                                                
##                                         opening_name   opening_ply   
##  Sicilian Defense                             : 334   Min.   : 1.00  
##  Van't Kruijs Opening                         : 327   1st Qu.: 2.00  
##  Sicilian Defense: Bowdler Attack             : 277   Median : 3.00  
##  French Defense: Knight Variation             : 246   Mean   : 3.38  
##  Scotch Game                                  : 243   3rd Qu.: 4.00  
##  Scandinavian Defense: Mieses-Kotroc Variation: 241   Max.   :11.00  
##  (Other)                                      :4552                  
##   initial_time    time_increment    win.bin 
##  Min.   :  0.00   Min.   :  0.000   0:3002  
##  1st Qu.: 10.00   1st Qu.:  0.000   1:3218  
##  Median : 10.00   Median :  0.000           
##  Mean   : 14.24   Mean   :  5.023           
##  3rd Qu.: 15.00   3rd Qu.:  7.000           
##  Max.   :180.00   Max.   :180.000           
## 
par(mfrow = c(2, 2))
hist(games$turns, breaks = 100)
hist(games$white_rating, breaks = 100)
hist(games$black_rating, breaks = 100)
hist(log(games$opening_ply), breaks = 100)

The frequencies of openings we’ve kept:

head(ocount, n = 10)
## # A tibble: 10 x 2
##    opening_name                                  count
##    <fct>                                         <int>
##  1 Sicilian Defense                                349
##  2 Van't Kruijs Opening                            342
##  3 Sicilian Defense: Bowdler Attack                290
##  4 French Defense: Knight Variation                260
##  5 Scotch Game                                     254
##  6 Scandinavian Defense: Mieses-Kotroc Variation   247
##  7 Queen's Pawn Game: Mason Attack                 227
##  8 Queen's Pawn Game: Chigorin Variation           217
##  9 Scandinavian Defense                            217
## 10 Horwitz Defense                                 208
barplot(ocount$count, main = "Frequencies of openings")

Training / testing data split

This is a large dataset, even after all the pruning of data we’ve done so far. We can afford to just set aside some (dedicated) data for testing.

I will use a 75/25 split for training/testing.

Note: I was planning to use double-cross validation for logistic regression (the neural network would take a prohibitive amount of time). I ran out of time and, with only a few hours left to the deadline, I am going to skip double-CV. At least the size of the dataset is in the thousands, so the split should offer a decent test. Double-CV is on my practice list, in R and Python, for this summer, before the fall semester. :)

dsn = dim(games)[1]
t.ratio = 0.75
train.size = round(dsn * t.ratio)
test.start = train.size + 1
d.train = games[1:train.size, ]
d.test  = games[test.start:dsn, ]

print("Training set size:", quote = F)
## [1] Training set size:
dim(d.train)[1]
## [1] 4665
print("Testing set size:", quote = F)
## [1] Testing set size:
dim(d.test)[1]
## [1] 1555

Plain logistic regression

Let’s train the regression model on the training slice of data:

set.seed(10)
f.log = glm(win.bin ~ ., data = d.train, family = "binomial")
summary(f.log)
## 
## Call:
## glm(formula = win.bin ~ ., family = "binomial", data = d.train)
## 
## Deviance Residuals: 
##     Min       1Q   Median       3Q      Max  
## -2.6469  -1.0423   0.3261   1.0298   2.8013  
## 
## Coefficients:
##                                                             Estimate Std. Error
## (Intercept)                                                0.8796688  0.3835904
## turns                                                     -0.0025493  0.0010131
## white_rating                                               0.0042032  0.0002009
## black_rating                                              -0.0044359  0.0001942
## opening_nameCaro-Kann Defense                             -0.1966227  0.2994505
## opening_nameFour Knights Game: Italian Variation           0.2222888  0.4667178
## opening_nameFrench Defense #2                             -0.4332493  0.3610534
## opening_nameFrench Defense: Exchange Variation             0.0959168  0.3903794
## opening_nameFrench Defense: Knight Variation              -0.1888099  0.2823831
## opening_nameFrench Defense: Normal Variation               0.0448043  0.3361073
## opening_nameGiuoco Piano                                  -0.5333089  0.4160872
## opening_nameHorwitz Defense                               -0.2423806  0.3024407
## opening_nameHungarian Opening                             -0.4992955  0.3623875
## opening_nameIndian Game                                   -0.6026959  0.3231523
## opening_nameItalian Game                                   0.0855543  0.3883187
## opening_nameItalian Game: Anti-Fried Liver Defense        -0.1420161  0.3835736
## opening_nameKing's Pawn Game: Leonardis Variation         -0.4434333  0.3053497
## opening_nameKing's Pawn Game: Wayward Queen Attack        -0.4646119  0.3039841
## opening_nameModern Defense                                -0.2406272  0.3167680
## opening_nameOwen Defense                                  -0.6111593  0.3161092
## opening_namePhilidor Defense                               0.0864527  0.3637600
## opening_namePhilidor Defense #2                           -0.3829342  0.3100417
## opening_namePhilidor Defense #3                            0.1762824  0.3400238
## opening_namePirc Defense #4                                0.1637602  0.3706851
## opening_nameQueen's Gambit Accepted: Old Variation        -0.2598652  0.3968254
## opening_nameQueen's Gambit Declined                       -0.0736039  0.3561222
## opening_nameQueen's Gambit Refused: Marshall Defense       0.3562995  0.3467057
## opening_nameQueen's Pawn                                   0.1728406  0.3545494
## opening_nameQueen's Pawn Game                             -0.6001289  0.3334554
## opening_nameQueen's Pawn Game #2                          -0.2461893  0.3429364
## opening_nameQueen's Pawn Game: Chigorin Variation         -0.1763997  0.2976340
## opening_nameQueen's Pawn Game: Mason Attack               -0.2113953  0.2874968
## opening_nameQueen's Pawn Game: Zukertort Variation        -0.3654132  0.3458986
## opening_nameRuy Lopez: Steinitz Defense                    0.0726829  0.4001390
## opening_nameScandinavian Defense                          -0.7046092  0.2963105
## opening_nameScandinavian Defense: Mieses-Kotroc Variation  0.0063835  0.2921826
## opening_nameScotch Game                                   -0.0896434  0.3659420
## opening_nameSicilian Defense                              -0.3958769  0.2718208
## opening_nameSicilian Defense: Bowdler Attack              -0.4052731  0.2813373
## opening_nameSicilian Defense: Old Sicilian                -0.6328364  0.3216639
## opening_nameSicilian Defense: Smith-Morra Gambit #2       -0.0002563  0.3321967
## opening_nameVan't Kruijs Opening                          -0.7158259  0.3192237
## opening_ply                                               -0.0187628  0.0777944
## initial_time                                              -0.0012262  0.0019967
## time_increment                                             0.0036287  0.0025812
##                                                           z value Pr(>|z|)    
## (Intercept)                                                 2.293   0.0218 *  
## turns                                                      -2.516   0.0119 *  
## white_rating                                               20.926   <2e-16 ***
## black_rating                                              -22.847   <2e-16 ***
## opening_nameCaro-Kann Defense                              -0.657   0.5114    
## opening_nameFour Knights Game: Italian Variation            0.476   0.6339    
## opening_nameFrench Defense #2                              -1.200   0.2302    
## opening_nameFrench Defense: Exchange Variation              0.246   0.8059    
## opening_nameFrench Defense: Knight Variation               -0.669   0.5037    
## opening_nameFrench Defense: Normal Variation                0.133   0.8940    
## opening_nameGiuoco Piano                                   -1.282   0.1999    
## opening_nameHorwitz Defense                                -0.801   0.4229    
## opening_nameHungarian Opening                              -1.378   0.1683    
## opening_nameIndian Game                                    -1.865   0.0622 .  
## opening_nameItalian Game                                    0.220   0.8256    
## opening_nameItalian Game: Anti-Fried Liver Defense         -0.370   0.7112    
## opening_nameKing's Pawn Game: Leonardis Variation          -1.452   0.1464    
## opening_nameKing's Pawn Game: Wayward Queen Attack         -1.528   0.1264    
## opening_nameModern Defense                                 -0.760   0.4475    
## opening_nameOwen Defense                                   -1.933   0.0532 .  
## opening_namePhilidor Defense                                0.238   0.8121    
## opening_namePhilidor Defense #2                            -1.235   0.2168    
## opening_namePhilidor Defense #3                             0.518   0.6042    
## opening_namePirc Defense #4                                 0.442   0.6587    
## opening_nameQueen's Gambit Accepted: Old Variation         -0.655   0.5126    
## opening_nameQueen's Gambit Declined                        -0.207   0.8363    
## opening_nameQueen's Gambit Refused: Marshall Defense        1.028   0.3041    
## opening_nameQueen's Pawn                                    0.487   0.6259    
## opening_nameQueen's Pawn Game                              -1.800   0.0719 .  
## opening_nameQueen's Pawn Game #2                           -0.718   0.4728    
## opening_nameQueen's Pawn Game: Chigorin Variation          -0.593   0.5534    
## opening_nameQueen's Pawn Game: Mason Attack                -0.735   0.4622    
## opening_nameQueen's Pawn Game: Zukertort Variation         -1.056   0.2908    
## opening_nameRuy Lopez: Steinitz Defense                     0.182   0.8559    
## opening_nameScandinavian Defense                           -2.378   0.0174 *  
## opening_nameScandinavian Defense: Mieses-Kotroc Variation   0.022   0.9826    
## opening_nameScotch Game                                    -0.245   0.8065    
## opening_nameSicilian Defense                               -1.456   0.1453    
## opening_nameSicilian Defense: Bowdler Attack               -1.441   0.1497    
## opening_nameSicilian Defense: Old Sicilian                 -1.967   0.0491 *  
## opening_nameSicilian Defense: Smith-Morra Gambit #2        -0.001   0.9994    
## opening_nameVan't Kruijs Opening                           -2.242   0.0249 *  
## opening_ply                                                -0.241   0.8094    
## initial_time                                               -0.614   0.5391    
## time_increment                                              1.406   0.1598    
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## (Dispersion parameter for binomial family taken to be 1)
## 
##     Null deviance: 6462.4  on 4664  degrees of freedom
## Residual deviance: 5500.0  on 4620  degrees of freedom
## AIC: 5590
## 
## Number of Fisher Scoring iterations: 4

Neural network

performance: the ratio of correct predictions / total predictions

We need to choose the best size and decay for the network, that will maximize performance.

The performance matrix, defined below, holds performance ratios at different decay values (rows) and different network sizes (columns).

The random seed affects performance greatly. Testing performance at many different seeds, then averaging the result, should provide a better picture. The code below does exactly that: it computes performance matrices using various seeds, then calculates the average matrix. The average is then used for choosing the best size and decay.

decay.list = seq(0.5, 5, by = 0.05)
size.list = seq(1, 16, by = 1)
seed.list = 1000:1047

The following block takes half a day to run on a reasonably new gaming PC, even with doParallel, so it has been disabled here. The end result of it is the perf-avg.csv file, which is included in the submitted files, and will be read in the next block, to make sure the whole notebook runs fine start-to-finish.

cl <- parallel::makeCluster(detectCores())
doParallel::registerDoParallel(cl)

for (tseed in seed.list) {
  # performance = ratio of correct predictions to total predictions
  perf = matrix(, nr = length(decay.list), nc = length(size.list))
  print(tseed)
  print(size.list)
  for (sz in size.list) {
    cat(" ", sz)
    perf.col = matrix(, nr = length(decay.list), nc = 1)
    perf.col <- foreach (dc = decay.list, .combine = rbind) %dopar% {
      set.seed(tseed)
      f.nn = nnet::nnet(win.bin ~ ., data = d.train, size = sz, MaxNWts = 100000, maxit = 10000, decay = dc, trace = F)
      stopifnot(f.nn$convergence == 0)
      tp.nn  = table(factor(predict(f.nn, d.test, type = "class"), levels = c(0, 1)), d.test$win.bin)
      perf.col[which(decay.list == dc), 1] = (tp.nn[1, 1] + tp.nn[2, 2]) / sum(tp.nn)
    }
    perf[, which(size.list == sz)] = perf.col
  }
  print("", quote = F)
  
  pfname = paste0("perf", tseed, ".csv")
  write.csv(perf, file = pfname)
}

parallel::stopCluster(cl)

perf = matrix(0, nr = length(decay.list), nc = length(size.list))

for (sd in seed.list) {
  pmf = paste0("perf", sd, ".csv")
  pm = as.matrix(read.csv(pmf))[, -1]
  
  phmf = paste0("frame-perf", sd, ".png")
  png(filename = phmf, width = 1600, height = 900)
  heatmap.2(exp(exp(exp(pm))) ^ 2, dendrogram = "none", Rowv = F, Colv = F, main = paste0("exp(exp(exp(perf))) ^ 2 at seed = ", sd))
  dev.off()
  
  perf = perf + pm
}
perf = perf / length(seed.list)

# animated loop showing performance at different seeds
# requires ImageMagick
shell("convert -delay 20 -loop 0 frame-perf*.png perf-anim.gif")

write.csv(perf, file = "perf-avg.csv", row.names = F)

Histogram of performance values from the averaged matrix generated above:

perf = as.matrix(read.csv("perf-avg.csv"))
hist(perf, breaks = 100, col = "lightblue", main = "Histogram of performance")

Performance heat map:

The horizontal axis shows sizes, the vertical axis shows decays. Each frame is a different seed value. Each cell contains the performance of the model: yellow is good, red is bad.

performance heat maps at different random seeds

Average performance heatmap over several dozen seed values - this is the average generated over all seed values:

heatmap.2(exp(exp(exp(perf))) ^ 2, dendrogram = "none", Rowv = F, Colv = F, main = "exp(exp(exp(perf))) ^ 2")

max.ind = which(perf == max(perf), arr.ind = T)
decay.best = max.ind[1]
size.best = max.ind[2]

print(paste("Best performance:", max(perf)), quote = F)
## [1] Best performance: 0.67591103965702
print(paste("Best decay:", decay.list[decay.best], "out of", min(decay.list), "to", max(decay.list)), quote = F)
## [1] Best decay: 3.55 out of 0.5 to 5
print(paste("Best size:", size.list[size.best], "out of", min(size.list), "to", max(size.list)), quote = F)
## [1] Best size: 2 out of 1 to 16

Averaging over seed values provided the best performance with a fairly parsimonious model (small size, high decay), so we’re going to keep these “best” values for size and decay.

Evaluate the model performance

Using the best size and the best decay, train the network on the training data slice. The logistic regression model is already trained.

set.seed(10)
f.nn = nnet(win.bin ~ ., data = d.train, size = size.list[size.best], MaxNWts = 100000, maxit = 10000, decay = decay.list[decay.best])
## # weights:  93
## initial  value 3303.669960 
## iter  10 value 3230.200256
## iter  20 value 3118.820454
## iter  30 value 2976.376020
## iter  40 value 2959.650678
## iter  50 value 2920.478791
## iter  60 value 2894.689497
## iter  70 value 2842.576596
## iter  80 value 2819.628814
## iter  90 value 2804.450625
## iter 100 value 2800.297132
## iter 110 value 2793.988489
## iter 120 value 2790.644013
## iter 130 value 2790.603701
## iter 140 value 2790.598186
## final  value 2790.598041 
## converged

Using both models, make predictions on the testing slice of the dataset, and compare with reality.

tp.log = table(predict(f.log, d.test, type = "response") > 0.5, d.test$win.bin)
tp.nn  = table(predict(f.nn, d.test, type = "class"), d.test$win.bin)

tp.log
##        
##           0   1
##   FALSE 476 235
##   TRUE  267 577
tp.nn
##    
##       0   1
##   0 474 230
##   1 269 582
perf.log = (tp.log[1, 1] + tp.log[2, 2]) / sum(tp.log)
perf.nn  = (tp.nn[1, 1] + tp.nn[2, 2]) / sum(tp.nn)

print("", quote = F)
## [1]
print(paste("Logistic regression performance:", perf.log), quote = F)
## [1] Logistic regression performance: 0.677170418006431
print(paste("Neural network performance:     ", perf.nn), quote = F)
## [1] Neural network performance:      0.679099678456592

ROC curves:

par(pty = 's')
plot(roc(predictor = predict(f.log, d.test, type = "response"), response = d.test$win.bin), col = "blue")
par(new = T)
plot(roc(predictor = predict(f.nn, d.test, type = "raw"), response = d.test$win.bin), col = "red")
legend("bottomright", legend = c("logistic regression", "neural network"), lty = c(1, 1), col = c("blue", "red"))

I was hoping for better performance, but it is what it is. The neural network performs slightly better.

We’re going to pick the neural network to make predictions.

Predictions

Train the neural network on all existing data:

model.nn = nnet(win.bin ~ ., data = games, size = size.list[size.best], MaxNWts = 100000, maxit = 10000, decay = decay.list[decay.best])
## # weights:  93
## initial  value 4357.327920 
## iter  10 value 3894.744310
## iter  20 value 3843.542165
## iter  30 value 3833.064106
## iter  40 value 3815.050081
## iter  50 value 3777.797957
## iter  60 value 3741.240091
## iter  70 value 3733.209817
## iter  80 value 3730.583319
## iter  90 value 3730.040393
## iter 100 value 3729.919077
## iter 110 value 3729.871542
## iter 120 value 3729.788072
## iter 130 value 3729.758219
## final  value 3729.757389 
## converged

The home team player has an ELO of 1500. They are going to play two games:

Which openings will maximize the chances of winning?

We will assume mean values for the other predictors (time, number of turns).

We will create one dataframe for each game; the only thing that changes from one row to another within each dataframe is the opening. Then we will use the model to see which opening is predicted to maximize the chances of winning.

elo.home = 1500
elo.p1   = 1600
elo.p2   = 1400

init.time.mean = exp(mean(log(games$initial_time), trim = 0.1))
time.incr.mean = exp(mean(log(games$time_increment[which(games$time_increment > 0)])))
o.ply.mean     = round(exp(mean(log(games$opening_ply[which(games$opening_ply > 0)]))))
turns.mean     = round(mean(games$turns))

op.tot = length(ocount$opening_name)

games.white = data.frame(
  turns = rep(turns.mean, times = op.tot),
  white_rating   = rep(elo.home, times = op.tot),
  black_rating   = rep(elo.p1, times = op.tot),
  opening_name   = ocount$opening_name,
  opening_ply    = rep(o.ply.mean, times = op.tot),
  initial_time   = rep(init.time.mean, times = op.tot),
  time_increment = rep(time.incr.mean, times = op.tot),
  win.bin        = as.factor(rep(1, times = op.tot))
)

games.black = data.frame(
  turns = rep(turns.mean, times = op.tot),
  white_rating   = rep(elo.p2, times = op.tot),
  black_rating   = rep(elo.home, times = op.tot),
  opening_name   = ocount$opening_name,
  opening_ply    = rep(o.ply.mean, times = op.tot),
  initial_time   = rep(init.time.mean, times = op.tot),
  time_increment = rep(time.incr.mean, times = op.tot),
  win.bin        = as.factor(rep(0, times = op.tot))
)

summary(games.white)
##      turns     white_rating   black_rating 
##  Min.   :59   Min.   :1500   Min.   :1600  
##  1st Qu.:59   1st Qu.:1500   1st Qu.:1600  
##  Median :59   Median :1500   Median :1600  
##  Mean   :59   Mean   :1500   Mean   :1600  
##  3rd Qu.:59   3rd Qu.:1500   3rd Qu.:1600  
##  Max.   :59   Max.   :1500   Max.   :1600  
##                                            
##                                opening_name  opening_ply  initial_time  
##  Bishop's Opening                    : 1    Min.   :3    Min.   :10.75  
##  Caro-Kann Defense                   : 1    1st Qu.:3    1st Qu.:10.75  
##  Four Knights Game: Italian Variation: 1    Median :3    Median :10.75  
##  French Defense #2                   : 1    Mean   :3    Mean   :10.75  
##  French Defense: Exchange Variation  : 1    3rd Qu.:3    3rd Qu.:10.75  
##  French Defense: Knight Variation    : 1    Max.   :3    Max.   :10.75  
##  (Other)                             :33                                
##  time_increment  win.bin
##  Min.   :7.386   1:39   
##  1st Qu.:7.386          
##  Median :7.386          
##  Mean   :7.386          
##  3rd Qu.:7.386          
##  Max.   :7.386          
## 
summary(games.black)
##      turns     white_rating   black_rating 
##  Min.   :59   Min.   :1400   Min.   :1500  
##  1st Qu.:59   1st Qu.:1400   1st Qu.:1500  
##  Median :59   Median :1400   Median :1500  
##  Mean   :59   Mean   :1400   Mean   :1500  
##  3rd Qu.:59   3rd Qu.:1400   3rd Qu.:1500  
##  Max.   :59   Max.   :1400   Max.   :1500  
##                                            
##                                opening_name  opening_ply  initial_time  
##  Bishop's Opening                    : 1    Min.   :3    Min.   :10.75  
##  Caro-Kann Defense                   : 1    1st Qu.:3    1st Qu.:10.75  
##  Four Knights Game: Italian Variation: 1    Median :3    Median :10.75  
##  French Defense #2                   : 1    Mean   :3    Mean   :10.75  
##  French Defense: Exchange Variation  : 1    3rd Qu.:3    3rd Qu.:10.75  
##  French Defense: Knight Variation    : 1    Max.   :3    Max.   :10.75  
##  (Other)                             :33                                
##  time_increment  win.bin
##  Min.   :7.386   0:39   
##  1st Qu.:7.386          
##  Median :7.386          
##  Mean   :7.386          
##  3rd Qu.:7.386          
##  Max.   :7.386          
## 

Predict the outcome of the game played as white against the stronger opponent. Here, prediction = 1 means highest odds of white winning, so we will sort the winning in decreasing order.

p.white = predict(model.nn, games.white, type = "raw")
pred.white = data.frame(
  opening  = games.white$opening_name,
  win_prob = as.vector(p.white)
)
best.white = pred.white[order(pred.white$win_prob, decreasing = T),]
write.csv(best.white, file = "best_white.csv", row.names = F)
best.white
##                                          opening  win_prob
## 25                                  Queen's Pawn 0.4501371
## 6  Scandinavian Defense: Mieses-Kotroc Variation 0.4377557
## 38                               Pirc Defense #4 0.4295530
## 34       Sicilian Defense: Smith-Morra Gambit #2 0.4198934
## 37                                  Italian Game 0.4196096
## 26      Queen's Gambit Refused: Marshall Defense 0.4164996
## 11                             Caro-Kann Defense 0.4140233
## 4               French Defense: Knight Variation 0.4138846
## 30                       Queen's Gambit Declined 0.4131132
## 23                              Philidor Defense 0.4128466
## 12                           Philidor Defense #3 0.4107184
## 28              French Defense: Normal Variation 0.3987685
## 31        Queen's Pawn Game: Zukertort Variation 0.3956471
## 18          Four Knights Game: Italian Variation 0.3919486
## 36            French Defense: Exchange Variation 0.3915099
## 32                          Queen's Pawn Game #2 0.3887330
## 10                               Horwitz Defense 0.3870948
## 27                              Bishop's Opening 0.3768875
## 8          Queen's Pawn Game: Chigorin Variation 0.3720320
## 13                           Philidor Defense #2 0.3713351
## 33        Queen's Gambit Accepted: Old Variation 0.3705794
## 21         King's Pawn Game: Leonardis Variation 0.3703631
## 24                   Ruy Lopez: Steinitz Defense 0.3631123
## 22                             Queen's Pawn Game 0.3624154
## 15                                Modern Defense 0.3617383
## 7                Queen's Pawn Game: Mason Attack 0.3616814
## 35                             French Defense #2 0.3595915
## 29                             Hungarian Opening 0.3589511
## 16        Italian Game: Anti-Fried Liver Defense 0.3533848
## 14                                   Indian Game 0.3499980
## 5                                    Scotch Game 0.3493635
## 3               Sicilian Defense: Bowdler Attack 0.3423866
## 17        King's Pawn Game: Wayward Queen Attack 0.3423066
## 19                                  Owen Defense 0.3383961
## 9                           Scandinavian Defense 0.3357196
## 1                               Sicilian Defense 0.3332519
## 20                Sicilian Defense: Old Sicilian 0.3257028
## 39                                  Giuoco Piano 0.3184512
## 2                           Van't Kruijs Opening 0.3121804

Predict the outcome of the game played as black against the weaker opponent. Here prediction = 0 means the highest odds of black winning, so we use sort in increasing order of winning.

p.black = predict(model.nn, games.black, type = "raw")
pred.black = data.frame(
  opening  = games.black$opening_name,
  win_prob = as.vector(p.black)
)
best.black = pred.black[order(pred.black$win_prob, decreasing = F),]
write.csv(best.black, file = "best_black.csv", row.names = F)
best.black
##                                          opening  win_prob
## 2                           Van't Kruijs Opening 0.3130230
## 39                                  Giuoco Piano 0.3193277
## 20                Sicilian Defense: Old Sicilian 0.3266225
## 1                               Sicilian Defense 0.3342153
## 9                           Scandinavian Defense 0.3366948
## 19                                  Owen Defense 0.3393778
## 17        King's Pawn Game: Wayward Queen Attack 0.3432906
## 3               Sicilian Defense: Bowdler Attack 0.3433769
## 5                                    Scotch Game 0.3503893
## 14                                   Indian Game 0.3510118
## 16        Italian Game: Anti-Fried Liver Defense 0.3544171
## 29                             Hungarian Opening 0.3600075
## 35                             French Defense #2 0.3606629
## 7                Queen's Pawn Game: Mason Attack 0.3627448
## 15                                Modern Defense 0.3628062
## 22                             Queen's Pawn Game 0.3634939
## 24                   Ruy Lopez: Steinitz Defense 0.3641951
## 21         King's Pawn Game: Leonardis Variation 0.3714704
## 33        Queen's Gambit Accepted: Old Variation 0.3716748
## 13                           Philidor Defense #2 0.3724496
## 8          Queen's Pawn Game: Chigorin Variation 0.3731463
## 27                              Bishop's Opening 0.3780086
## 10                               Horwitz Defense 0.3882349
## 32                          Queen's Pawn Game #2 0.3898873
## 36            French Defense: Exchange Variation 0.3926739
## 18          Four Knights Game: Italian Variation 0.3931091
## 31        Queen's Pawn Game: Zukertort Variation 0.3968183
## 28              French Defense: Normal Variation 0.3999505
## 12                           Philidor Defense #3 0.4119202
## 23                              Philidor Defense 0.4140498
## 30                       Queen's Gambit Declined 0.4143256
## 4               French Defense: Knight Variation 0.4151011
## 11                             Caro-Kann Defense 0.4152337
## 26      Queen's Gambit Refused: Marshall Defense 0.4177115
## 37                                  Italian Game 0.4208370
## 34       Sicilian Defense: Smith-Morra Gambit #2 0.4211184
## 38                               Pirc Defense #4 0.4307895
## 6  Scandinavian Defense: Mieses-Kotroc Variation 0.4389943
## 25                                  Queen's Pawn 0.4513830

TODO

I am not sure whether the model’s predictions are stable when the other variables change. E.g., would it be better to aim for a longer game, or shorter?